In [6]:
cv2_imshow(images[0])
by Michele Casalgrandi
The Aarhus University Signal Processing group released a dataset of images of unique plants belonging to 12 species at several growth stages.
Import needed libraries
import numpy as np
import math
import pandas as pd
import random as rnd
import matplotlib.pyplot as plt
import cv2
from scipy import stats
from scipy.signal import convolve2d # Convolve two 2-dimensional arrays.
%matplotlib inline
import seaborn as sns
# rnd.seed = 1
from tensorflow.keras.utils import to_categorical
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, classification_report
from google.colab.patches import cv2_imshow # cv2.imshow() crashes colab
from tensorflow.keras.initializers import he_normal, he_uniform, GlorotNormal, GlorotUniform, LecunNormal
Load images and labels
from google.colab import drive
drive.mount('/content/drive')
Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).
path = "/content/drive/MyDrive/data/"
images = np.load(path + '/' + 'images.npy')
print(type(images))
print(images.shape)
labels_df = pd.read_csv(path + '/' + 'Labels.csv')
labels = labels_df.to_numpy()
print(labels.shape)
<class 'numpy.ndarray'> (4750, 128, 128, 3) (4750, 1)
The dataset contains 4750 images of size 128x128 and 3 channels
print('Min pixel intensity:', images.min())
print('Max pixel intensity:', images.max())
print('Pixel intensity mean:', round(images.mean(),1))
print('Pixel intensity std deviation:', round(images.std(), 1))
Min pixel intensity: 0 Max pixel intensity: 255 Pixel intensity mean: 70.0 Pixel intensity std deviation: 32.0
# define function to plot a grid of pictures
from mpl_toolkits.axes_grid1 import ImageGrid
def plot_image_samples(size, images, labels, indexes=None):
if not indexes:
indexes = rnd.sample(range(0,images.shape[0]), size[0] * size[1])
fig = plt.figure(figsize=(size[1] * 3, size[0] * 3))
grid = ImageGrid(fig, 111, nrows_ncols=size,axes_pad=0.4)
for ax, im, label in zip(grid, images[indexes], labels[indexes]):
ax.imshow(cv2.cvtColor(im, cv2.COLOR_BGR2RGB))
ax.set_title(label)
plt.show()
cv2_imshow(images[0])
plt.imshow(images[0])
<matplotlib.image.AxesImage at 0x7f7426f3ce90>
Images are stored as BGR - we will convert to RGB so we can use matplotlib imshow()
# initialize an array with same shape as original
images_RGB = np.zeros(images.shape)
images_RGB.shape
(4750, 128, 128, 3)
# convert to RGB
for idx, im in enumerate(images):
images_RGB[idx] = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
images_RGB = images_RGB.astype('int')
plt.imshow(cv2.cvtColor(images[0], cv2.COLOR_BGR2RGB))
<matplotlib.image.AxesImage at 0x7f7425247490>
# print stats for original and RGB array
print('Original array:', 'Mean:',round(images.mean(),2),'Min:',images.min(),'Max:',images.max())
print('RGB array:', 'Mean:',round(images_RGB.mean(),2),'Min:',images_RGB.min(),'Max:',images_RGB.max())
Original array: Mean: 70.04 Min: 0 Max: 255 RGB array: Mean: 70.04 Min: 0 Max: 255
plt.imshow(images_RGB[0])
<matplotlib.image.AxesImage at 0x7f74251c24d0>
Select random sample from the images
# select a sample of the pictures
grid_size = (2,6)
sample_images_idx = rnd.sample(range(0,images_RGB.shape[0]), grid_size[0] * grid_size[1])
print(sample_images_idx)
[4321, 3478, 2469, 212, 3769, 806, 3041, 4014, 41, 2011, 3140, 3030]
# define function to plot a grid of pictures
from mpl_toolkits.axes_grid1 import ImageGrid
def plot_image_samples(size, images, labels, indexes=None):
if not indexes:
indexes = rnd.sample(range(0,images.shape[0]), size[0] * size[1])
fig = plt.figure(figsize=(size[1] * 3, size[0] * 3))
grid = ImageGrid(fig, 111, nrows_ncols=size,axes_pad=0.4)
for ax, im, label in zip(grid, images[indexes], labels[indexes]):
ax.imshow(im)
ax.set_title(label)
plt.show()
# plot the sample
plot_image_samples(grid_size, images_RGB, labels, sample_images_idx)
# define function to plot the images histograms
def plot_histograms(size, images, labels, indexes):
rows = size[0]
cols = size[1]
fig = plt.figure(figsize=(cols * 3, rows * 3))
idx=0
bins=20
xrange=(images.min(),images.max())
for row in range(rows):
for col in range(cols):
plt.subplot(rows, cols, idx+1)
plt.hist(images[indexes[idx]].flatten(), bins=60, range=xrange, density=True, stacked=False)
fig.x_title=labels[indexes[idx]]
idx += 1
plt.show()
# plot the histograms
plot_histograms(grid_size, images_RGB, labels, sample_images_idx)
The pixel intensities are skewed to the right, indicating the images tend to be dark.
The narrow shape of the distribution indicates they are low contrast.
Images with the ruler have a second peak due to the bright white areas of the ruler.
labels_df
| Label | |
|---|---|
| 0 | Small-flowered Cranesbill |
| 1 | Small-flowered Cranesbill |
| 2 | Small-flowered Cranesbill |
| 3 | Small-flowered Cranesbill |
| 4 | Small-flowered Cranesbill |
| ... | ... |
| 4745 | Loose Silky-bent |
| 4746 | Loose Silky-bent |
| 4747 | Loose Silky-bent |
| 4748 | Loose Silky-bent |
| 4749 | Loose Silky-bent |
4750 rows × 1 columns
# print out unique values of the labels
labels_counts = labels_df['Label'].value_counts()
print(labels_counts)
labels_ordered = labels_counts.index.to_list()
Loose Silky-bent 654 Common Chickweed 611 Scentless Mayweed 516 Small-flowered Cranesbill 496 Fat Hen 475 Charlock 390 Sugar beet 385 Cleavers 287 Black-grass 263 Shepherds Purse 231 Maize 221 Common wheat 221 Name: Label, dtype: int64
print('There are', len(labels_ordered), 'levels:',labels_ordered)
There are 12 levels: ['Loose Silky-bent', 'Common Chickweed', 'Scentless Mayweed', 'Small-flowered Cranesbill', 'Fat Hen', 'Charlock', 'Sugar beet', 'Cleavers', 'Black-grass', 'Shepherds Purse', 'Maize', 'Common wheat']
The classes are imbalanced.
E.g. There are roughly three time as many 'Loose Silky-bent' as 'Common wheat'
# plot graph sorted by count
fig = plt.figure(figsize=(20, 5))
plt.xticks(rotation=45)
sns.countplot(x=labels_df['Label'], order=labels_ordered)
<matplotlib.axes._subplots.AxesSubplot at 0x7f7424d7f4d0>
# get the list of indexes for each category so we can select from them
images_by_category = []
for categ in labels_ordered:
categ_indexes = labels_df[labels_df['Label'] == categ].index.to_list()
images_by_category.append(categ_indexes)
# show images for each category
grid_size_categ = (1,8)
for categ in labels_ordered:
# total images = rows * columsn
img_count = grid_size_categ[0]*grid_size_categ[1]
# get position in array of category
categ_idx = labels_ordered.index(categ)
# get random samples from that category
categ_indexes = rnd.sample(images_by_category[categ_idx], img_count)
# plot images
plot_image_samples(grid_size_categ, images_RGB, labels, categ_indexes)
# plot histograms
plot_histograms(grid_size_categ, images_RGB, labels, categ_indexes)
print('Mean', images_RGB.mean())
print('Min', images_RGB.min())
print('Max', images_RGB.max())
Mean 70.04363745545504 Min 0 Max 255
# normalize to range 0, 1
images_post_prep = images_RGB / 255
print('Mean', images_post_prep.mean())
print('Min', images_post_prep.min())
print('Max', images_post_prep.max())
Mean 0.2746809311978636 Min 0.0 Max 1.0
images_post_prep.shape
(4750, 128, 128, 3)
# plot the sample after normalization
plot_image_samples(grid_size, images_post_prep, labels, sample_images_idx)
Resize one image and test gaussian blur.
IMAGE_SIZE = (64, 64) # target size of image
# resize the first image, index = 0
image = cv2.resize(images_post_prep[0], dsize=IMAGE_SIZE)
# render original
plt.imshow(images_post_prep[0])
<matplotlib.image.AxesImage at 0x7f7424c0ba10>
# render resized image
plt.imshow(image)
<matplotlib.image.AxesImage at 0x7f7424ac3510>
# blur with 5 by 5 kernel
kernel_size = (5,5)
image_blur = cv2.GaussianBlur(image, kernel_size, 1, 1)
# render blurred image
plt.imshow(image_blur)
<matplotlib.image.AxesImage at 0x7f7424a77f50>
Resize and blur the entire array of images
# initialize an array with same shape as source
images_resize_blur = np.zeros((images_post_prep.shape[0],) + IMAGE_SIZE + (3,))
images_resize_blur.shape
(4750, 64, 64, 3)
# check type
images_resize_blur.dtype
dtype('float64')
# resize the images and blur
for idx, img in enumerate(images_post_prep):
blurred = cv2.GaussianBlur(
cv2.resize(img, IMAGE_SIZE),
kernel_size, 1, 1)
images_resize_blur[idx] = blurred
# check new shape
images_resize_blur.shape
(4750, 64, 64, 3)
# view a small sample of the images
plot_image_samples((1,4), images_resize_blur, labels, range(4))
plot_histograms((1,4), images_resize_blur, labels, range(4))
View the images for each category after resizing and blurring
# show images for each category
grid_size_categ = (1,6)
for categ in labels_ordered:
img_count = grid_size_categ[0]*grid_size_categ[1]
categ_idx = labels_ordered.index(categ)
categ_indexes = rnd.sample(images_by_category[categ_idx], img_count)
plot_image_samples(grid_size_categ, images_resize_blur, labels, categ_indexes)
plot_histograms(grid_size_categ, images_resize_blur, labels, categ_indexes)
from sklearn.preprocessing import OneHotEncoder
# get labels encoded
encoder = OneHotEncoder(sparse=False)
y = encoder.fit_transform(labels)
# list encoded categories
encoder.categories_
[array(['Black-grass', 'Charlock', 'Cleavers', 'Common Chickweed',
'Common wheat', 'Fat Hen', 'Loose Silky-bent', 'Maize',
'Scentless Mayweed', 'Shepherds Purse',
'Small-flowered Cranesbill', 'Sugar beet'], dtype=object)]
Print label for the first image (index = 0)
image_index = 0
print('First image label:', labels_df['Label'][image_index])
print('Label encoded:', y[image_index])
print('Label inverse encode:', encoder.inverse_transform([y[0]])[0][image_index])
First image label: Small-flowered Cranesbill Label encoded: [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0.] Label inverse encode: Small-flowered Cranesbill
# split into train and val+test
X_train, X_temp, y_train, y_temp = train_test_split(images_resize_blur, y, test_size=0.3, stratify=labels)
# split val+test into validation and test
X_test, X_val, y_test, y_val = train_test_split(X_temp, y_temp, test_size=0.5, stratify=y_temp)
# check shapes are as expected
print('Train shapes:', X_train.shape, y_train.shape)
print('Validation shapes:', X_val.shape, y_val.shape)
print('Test shapes:', X_test.shape, y_test.shape)
Train shapes: (3325, 64, 64, 3) (3325, 12) Validation shapes: (713, 64, 64, 3) (713, 12) Test shapes: (712, 64, 64, 3) (712, 12)
# check class frequencies across data sets
print('Original dataset class frequencies: ', np.round(np.sum(y, axis=0)/np.sum(y), 3) * 100)
print('Train dataset class frequencies: ', np.round(np.sum(y_train, axis=0)/np.sum(y_train), 3) * 100)
print('Validation dataset class frequencies:', np.round(np.sum(y_val, axis=0)/np.sum(y_val), 3) * 100)
print('Test dataset class frequencies: ', np.round(np.sum(y_test, axis=0)/np.sum(y_test), 3) * 100)
Original dataset class frequencies: [ 5.5 8.2 6. 12.9 4.7 10. 13.8 4.7 10.9 4.9 10.4 8.1] Train dataset class frequencies: [ 5.5 8.2 6. 12.9 4.7 10. 13.8 4.7 10.9 4.9 10.4 8.1] Validation dataset class frequencies: [ 5.5 8.1 6. 12.9 4.6 10.1 13.7 4.6 10.9 4.8 10.5 8.1] Test dataset class frequencies: [ 5.6 8.3 6. 12.8 4.6 10. 13.8 4.6 10.8 4.9 10.4 8.1]
# compute class weights as the inverse of their frequency in the dataset
test_class_weights = 1 / (np.sum(y_train, axis=0)/np.sum(y_train) * 100)
weights = dict()
for idx, weight in enumerate(test_class_weights):
weights[idx] = weight
print(weights)
{0: 0.18070652173913043, 1: 0.12179487179487178, 2: 0.1654228855721393, 3: 0.07768691588785048, 4: 0.21451612903225806, 5: 0.10015060240963855, 6: 0.07259825327510917, 7: 0.21451612903225806, 8: 0.09210526315789473, 9: 0.2052469135802469, 10: 0.09582132564841499, 11: 0.12360594795539034}
The data can be used as is for modeling in keras so there is no need to reshape.
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, MaxPool2D, AvgPool2D, BatchNormalization, Dropout, Flatten, LeakyReLU
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.losses import categorical_crossentropy
from tensorflow.keras.metrics import Accuracy, CategoricalAccuracy
We will use the 'Adam' optimizer and 'categorical_crossentropy' as loss function by default.
# function to create model based on parameter 'layers'
def create_model(layers, opt=Adam(), lossf=categorical_crossentropy, metrics=[CategoricalAccuracy()]):
# neural network
model = Sequential()
for layer in layers:
model.add(layer)
# compile the model
model.compile(optimizer=opt,
loss=lossf,
metrics=metrics
)
return model
# define layers and create a model
input_shape = X_train.shape[1:]
print(input_shape)
layers = [
Conv2D(filters=32, kernel_size=3, activation='relu', input_shape=input_shape),
Conv2D(filters=64, kernel_size=3, activation='relu'),
MaxPool2D(pool_size=(2,2), strides=(2,2)),
Conv2D(filters=128, kernel_size=3, activation='relu'),
Flatten(),
Dense(units=500, activation='relu'),
Dense(units=12, activation='softmax')
]
model = create_model(layers)
model.summary()
(64, 64, 3) Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d (Conv2D) (None, 62, 62, 32) 896 _________________________________________________________________ conv2d_1 (Conv2D) (None, 60, 60, 64) 18496 _________________________________________________________________ max_pooling2d (MaxPooling2D) (None, 30, 30, 64) 0 _________________________________________________________________ conv2d_2 (Conv2D) (None, 28, 28, 128) 73856 _________________________________________________________________ flatten (Flatten) (None, 100352) 0 _________________________________________________________________ dense (Dense) (None, 500) 50176500 _________________________________________________________________ dense_1 (Dense) (None, 12) 6012 ================================================================= Total params: 50,275,760 Trainable params: 50,275,760 Non-trainable params: 0 _________________________________________________________________
# model.fit(X_train, y_train, epochs=3, batch_size=100, validation_data=(X_val, y_val))
history = model.fit(X_train, y_train, epochs=20, batch_size=100, validation_data=(X_val, y_val))
Epoch 1/20 34/34 [==============================] - 3s 38ms/step - loss: 2.3439 - categorical_accuracy: 0.2129 - val_loss: 1.8064 - val_categorical_accuracy: 0.3548 Epoch 2/20 34/34 [==============================] - 1s 29ms/step - loss: 1.5721 - categorical_accuracy: 0.4448 - val_loss: 1.4418 - val_categorical_accuracy: 0.4867 Epoch 3/20 34/34 [==============================] - 1s 29ms/step - loss: 1.2641 - categorical_accuracy: 0.5504 - val_loss: 1.2287 - val_categorical_accuracy: 0.5666 Epoch 4/20 34/34 [==============================] - 1s 29ms/step - loss: 1.0412 - categorical_accuracy: 0.6400 - val_loss: 1.0436 - val_categorical_accuracy: 0.6508 Epoch 5/20 34/34 [==============================] - 1s 29ms/step - loss: 0.8818 - categorical_accuracy: 0.7035 - val_loss: 1.0557 - val_categorical_accuracy: 0.6227 Epoch 6/20 34/34 [==============================] - 1s 29ms/step - loss: 0.7665 - categorical_accuracy: 0.7377 - val_loss: 0.9614 - val_categorical_accuracy: 0.6942 Epoch 7/20 34/34 [==============================] - 1s 29ms/step - loss: 0.6179 - categorical_accuracy: 0.7979 - val_loss: 1.0201 - val_categorical_accuracy: 0.6760 Epoch 8/20 34/34 [==============================] - 1s 29ms/step - loss: 0.5100 - categorical_accuracy: 0.8310 - val_loss: 0.9450 - val_categorical_accuracy: 0.7139 Epoch 9/20 34/34 [==============================] - 1s 29ms/step - loss: 0.3836 - categorical_accuracy: 0.8746 - val_loss: 1.0303 - val_categorical_accuracy: 0.6942 Epoch 10/20 34/34 [==============================] - 1s 29ms/step - loss: 0.3311 - categorical_accuracy: 0.8911 - val_loss: 1.0128 - val_categorical_accuracy: 0.7027 Epoch 11/20 34/34 [==============================] - 1s 29ms/step - loss: 0.2615 - categorical_accuracy: 0.9137 - val_loss: 1.0655 - val_categorical_accuracy: 0.7209 Epoch 12/20 34/34 [==============================] - 1s 29ms/step - loss: 0.1866 - categorical_accuracy: 0.9365 - val_loss: 1.1017 - val_categorical_accuracy: 0.7153 Epoch 13/20 34/34 [==============================] - 1s 29ms/step - loss: 0.1560 - categorical_accuracy: 0.9540 - val_loss: 1.1235 - val_categorical_accuracy: 0.7055 Epoch 14/20 34/34 [==============================] - 1s 29ms/step - loss: 0.0935 - categorical_accuracy: 0.9753 - val_loss: 1.4214 - val_categorical_accuracy: 0.7237 Epoch 15/20 34/34 [==============================] - 1s 29ms/step - loss: 0.0629 - categorical_accuracy: 0.9841 - val_loss: 1.3353 - val_categorical_accuracy: 0.7237 Epoch 16/20 34/34 [==============================] - 1s 29ms/step - loss: 0.0433 - categorical_accuracy: 0.9898 - val_loss: 1.4721 - val_categorical_accuracy: 0.7083 Epoch 17/20 34/34 [==============================] - 1s 29ms/step - loss: 0.0275 - categorical_accuracy: 0.9931 - val_loss: 1.5601 - val_categorical_accuracy: 0.6872 Epoch 18/20 34/34 [==============================] - 1s 29ms/step - loss: 0.0167 - categorical_accuracy: 0.9964 - val_loss: 1.5669 - val_categorical_accuracy: 0.7209 Epoch 19/20 34/34 [==============================] - 1s 29ms/step - loss: 0.0118 - categorical_accuracy: 0.9976 - val_loss: 1.7692 - val_categorical_accuracy: 0.7153 Epoch 20/20 34/34 [==============================] - 1s 29ms/step - loss: 0.0053 - categorical_accuracy: 0.9997 - val_loss: 1.7286 - val_categorical_accuracy: 0.7237
# create function to plot the loss against epochs
def plot_loss_by_epoch(history):
# put history in a dataframe
hist = pd.DataFrame(history.history)
hist['epoch'] = history.epoch
# Plotting loss at different epochs
plt.plot(hist['loss'])
plt.plot(hist['val_loss'])
plt.legend(("train" , "valid") , loc =0)
plt.show()
# plot the loss
plot_loss_by_epoch(history)
# evaluate the model
acc_model = model.evaluate(X_val, y_val)
acc_model
23/23 [==============================] - 0s 6ms/step - loss: 1.7286 - categorical_accuracy: 0.7237
[1.7285555601119995, 0.7237026691436768]
# create a dataframe to hold the scores so we can track performance across models
scores_columns = ['Model', 'Accuracy Validation']
scores = pd.DataFrame([['conv32|64|pool|conv128|dense500', acc_model[1]]],
columns=scores_columns)
scores
| Model | Accuracy Validation | |
|---|---|---|
| 0 | conv32|64|pool|conv128|dense500 | 0.723703 |
Accuracy for validation set is low.
The model is overfit as shown by the rise in validation loss after the initial drop.
Using a higher epoch number will not help.
# define function to plot confusion matrix
def plot_confusion_matrix(clssf, X, y_actual, threshold=0.5):
# get predictions from classifier and decode
val_predict = (clssf.predict(X) > threshold ) * 1
# decode labels
val_labels = encoder.inverse_transform(y_actual)
pred_labels = encoder.inverse_transform(val_predict)
# get confusion matrix
cm = confusion_matrix(val_labels, pred_labels)
# get normalized confusion matrix
cm_norm = confusion_matrix(val_labels, pred_labels, normalize='true')
# plot confusion matrix
plt.figure(figsize=(14,12))
# add normalized rates and predictions counts
annotations = np.array([f'{norm}\n({count})' for count, norm in zip(cm.flatten(), np.round(cm_norm,2).flatten())]).reshape((12,12))
# plot
ax = sns.heatmap(cm_norm, square=True, annot=annotations, fmt='', xticklabels=encoder.categories_[0],yticklabels=encoder.categories_[0], cbar=False, cmap='coolwarm')
plt.xticks(rotation=45)
plt.show()
print(classification_report(val_labels, pred_labels))
# plot confusion matrix for validation set
plot_confusion_matrix(model, X_val, y_val)
precision recall f1-score support
Black-grass 0.17 0.15 0.16 39
Charlock 0.85 0.88 0.86 58
Cleavers 0.84 0.74 0.79 43
Common Chickweed 0.89 0.85 0.87 92
Common wheat 0.44 0.58 0.50 33
Fat Hen 0.78 0.78 0.78 72
Loose Silky-bent 0.67 0.70 0.69 98
Maize 0.57 0.48 0.52 33
Scentless Mayweed 0.68 0.81 0.74 78
Shepherds Purse 0.70 0.56 0.62 34
Small-flowered Cranesbill 0.88 0.84 0.86 75
Sugar beet 0.76 0.71 0.73 58
accuracy 0.72 713
macro avg 0.69 0.67 0.68 713
weighted avg 0.72 0.72 0.72 713
Black-grass is misclassified as Loose Silky-bent at a higher rate than the correct classificationLoose Silky-bent is often misclassified as Black-grassCommon wheat is often misclassified as Black-grassCharlock and Small-flowered Cranesbill have the highest rate of true positives(This is listed as part of the assigment in the pdf version of the project)
print('Test dataset shape', X_test.shape)
Test dataset shape (712, 64, 64, 3)
# list of indexes as per assignment pdf instructions
prediction_indexes = [2, 3, 33, 36, 59]
predictions = (model.predict(X_test[prediction_indexes]) > 0.5) * 1
predicted_labels = encoder.inverse_transform(predictions)
actual_labels = encoder.inverse_transform(y_test[prediction_indexes])
# plot images
for img, pred, actual in zip(X_test[prediction_indexes], predicted_labels, actual_labels):
print('\n================================\n')
print('Actual label:', actual)
print('Predicted label:', pred)
plt.imshow(img)
plt.show()
================================ Actual label: ['Scentless Mayweed'] Predicted label: ['Scentless Mayweed']
================================ Actual label: ['Black-grass'] Predicted label: ['Loose Silky-bent']
================================ Actual label: ['Common wheat'] Predicted label: ['Loose Silky-bent']
================================ Actual label: ['Common wheat'] Predicted label: ['Common wheat']
================================ Actual label: ['Sugar beet'] Predicted label: ['Fat Hen']
input_shape = X_train.shape[1:]
print(input_shape)
# same layers but with padding = 'same'
layers = [
Conv2D(filters=32, kernel_size=3, activation='relu', input_shape=input_shape, padding='same'),
Conv2D(filters=64, kernel_size=3, activation='relu', padding='same'),
MaxPool2D(pool_size=(2,2), strides=(2,2)),
Conv2D(filters=128, kernel_size=3, activation='relu'),
Flatten(),
Dense(units=500, activation='relu'),
Dense(units=12, activation='softmax')
]
model2 = create_model(layers)
model2.summary()
(64, 64, 3) Model: "sequential_1" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_3 (Conv2D) (None, 64, 64, 32) 896 _________________________________________________________________ conv2d_4 (Conv2D) (None, 64, 64, 64) 18496 _________________________________________________________________ max_pooling2d_1 (MaxPooling2 (None, 32, 32, 64) 0 _________________________________________________________________ conv2d_5 (Conv2D) (None, 30, 30, 128) 73856 _________________________________________________________________ flatten_1 (Flatten) (None, 115200) 0 _________________________________________________________________ dense_2 (Dense) (None, 500) 57600500 _________________________________________________________________ dense_3 (Dense) (None, 12) 6012 ================================================================= Total params: 57,699,760 Trainable params: 57,699,760 Non-trainable params: 0 _________________________________________________________________
# fit the model
history2 = model2.fit(X_train, y_train, epochs=40, batch_size=100, validation_data=(X_val, y_val))
Epoch 1/40 34/34 [==============================] - 2s 40ms/step - loss: 3.4456 - categorical_accuracy: 0.2397 - val_loss: 2.4204 - val_categorical_accuracy: 0.2118 Epoch 2/40 34/34 [==============================] - 1s 31ms/step - loss: 2.3542 - categorical_accuracy: 0.2036 - val_loss: 2.2785 - val_categorical_accuracy: 0.2146 Epoch 3/40 34/34 [==============================] - 1s 31ms/step - loss: 2.0351 - categorical_accuracy: 0.3023 - val_loss: 1.9107 - val_categorical_accuracy: 0.3114 Epoch 4/40 34/34 [==============================] - 1s 31ms/step - loss: 1.7745 - categorical_accuracy: 0.3859 - val_loss: 1.7254 - val_categorical_accuracy: 0.3955 Epoch 5/40 34/34 [==============================] - 1s 31ms/step - loss: 1.6337 - categorical_accuracy: 0.4292 - val_loss: 1.6700 - val_categorical_accuracy: 0.3955 Epoch 6/40 34/34 [==============================] - 1s 31ms/step - loss: 1.5818 - categorical_accuracy: 0.4532 - val_loss: 1.6017 - val_categorical_accuracy: 0.4376 Epoch 7/40 34/34 [==============================] - 1s 31ms/step - loss: 1.5263 - categorical_accuracy: 0.4577 - val_loss: 1.6566 - val_categorical_accuracy: 0.4544 Epoch 8/40 34/34 [==============================] - 1s 31ms/step - loss: 1.4870 - categorical_accuracy: 0.4803 - val_loss: 1.5337 - val_categorical_accuracy: 0.4909 Epoch 9/40 34/34 [==============================] - 1s 31ms/step - loss: 1.4289 - categorical_accuracy: 0.5050 - val_loss: 1.5202 - val_categorical_accuracy: 0.4670 Epoch 10/40 34/34 [==============================] - 1s 31ms/step - loss: 1.4021 - categorical_accuracy: 0.5122 - val_loss: 1.5382 - val_categorical_accuracy: 0.4979 Epoch 11/40 34/34 [==============================] - 1s 31ms/step - loss: 1.3744 - categorical_accuracy: 0.5206 - val_loss: 1.4857 - val_categorical_accuracy: 0.4895 Epoch 12/40 34/34 [==============================] - 1s 31ms/step - loss: 1.3485 - categorical_accuracy: 0.5341 - val_loss: 1.4893 - val_categorical_accuracy: 0.4951 Epoch 13/40 34/34 [==============================] - 1s 31ms/step - loss: 1.3078 - categorical_accuracy: 0.5423 - val_loss: 1.4617 - val_categorical_accuracy: 0.5133 Epoch 14/40 34/34 [==============================] - 1s 31ms/step - loss: 1.2940 - categorical_accuracy: 0.5558 - val_loss: 1.4747 - val_categorical_accuracy: 0.5035 Epoch 15/40 34/34 [==============================] - 1s 31ms/step - loss: 1.2829 - categorical_accuracy: 0.5507 - val_loss: 1.4052 - val_categorical_accuracy: 0.5456 Epoch 16/40 34/34 [==============================] - 1s 31ms/step - loss: 1.2280 - categorical_accuracy: 0.5814 - val_loss: 1.3992 - val_categorical_accuracy: 0.5119 Epoch 17/40 34/34 [==============================] - 1s 31ms/step - loss: 1.2060 - categorical_accuracy: 0.5792 - val_loss: 1.4270 - val_categorical_accuracy: 0.5147 Epoch 18/40 34/34 [==============================] - 1s 31ms/step - loss: 1.2048 - categorical_accuracy: 0.5841 - val_loss: 1.3913 - val_categorical_accuracy: 0.5316 Epoch 19/40 34/34 [==============================] - 1s 31ms/step - loss: 1.1758 - categorical_accuracy: 0.5871 - val_loss: 1.4343 - val_categorical_accuracy: 0.5119 Epoch 20/40 34/34 [==============================] - 1s 31ms/step - loss: 1.1932 - categorical_accuracy: 0.5838 - val_loss: 1.3501 - val_categorical_accuracy: 0.5540 Epoch 21/40 34/34 [==============================] - 1s 31ms/step - loss: 1.1296 - categorical_accuracy: 0.6081 - val_loss: 1.3508 - val_categorical_accuracy: 0.5806 Epoch 22/40 34/34 [==============================] - 1s 31ms/step - loss: 1.1005 - categorical_accuracy: 0.6186 - val_loss: 1.3592 - val_categorical_accuracy: 0.5568 Epoch 23/40 34/34 [==============================] - 1s 31ms/step - loss: 1.0847 - categorical_accuracy: 0.6286 - val_loss: 1.3793 - val_categorical_accuracy: 0.5442 Epoch 24/40 34/34 [==============================] - 1s 31ms/step - loss: 1.1057 - categorical_accuracy: 0.6171 - val_loss: 1.3159 - val_categorical_accuracy: 0.5750 Epoch 25/40 34/34 [==============================] - 1s 31ms/step - loss: 1.0429 - categorical_accuracy: 0.6394 - val_loss: 1.3648 - val_categorical_accuracy: 0.5750 Epoch 26/40 34/34 [==============================] - 1s 31ms/step - loss: 1.0518 - categorical_accuracy: 0.6286 - val_loss: 1.3381 - val_categorical_accuracy: 0.5540 Epoch 27/40 34/34 [==============================] - 1s 31ms/step - loss: 1.0011 - categorical_accuracy: 0.6514 - val_loss: 1.3533 - val_categorical_accuracy: 0.5694 Epoch 28/40 34/34 [==============================] - 1s 31ms/step - loss: 0.9952 - categorical_accuracy: 0.6565 - val_loss: 1.4041 - val_categorical_accuracy: 0.5596 Epoch 29/40 34/34 [==============================] - 1s 31ms/step - loss: 0.9849 - categorical_accuracy: 0.6683 - val_loss: 1.3621 - val_categorical_accuracy: 0.5512 Epoch 30/40 34/34 [==============================] - 1s 31ms/step - loss: 0.9491 - categorical_accuracy: 0.6683 - val_loss: 1.3254 - val_categorical_accuracy: 0.5652 Epoch 31/40 34/34 [==============================] - 1s 31ms/step - loss: 0.9223 - categorical_accuracy: 0.6749 - val_loss: 1.3324 - val_categorical_accuracy: 0.5652 Epoch 32/40 34/34 [==============================] - 1s 31ms/step - loss: 0.8777 - categorical_accuracy: 0.6980 - val_loss: 1.3519 - val_categorical_accuracy: 0.5778 Epoch 33/40 34/34 [==============================] - 1s 31ms/step - loss: 0.8405 - categorical_accuracy: 0.7065 - val_loss: 1.1892 - val_categorical_accuracy: 0.6283 Epoch 34/40 34/34 [==============================] - 1s 31ms/step - loss: 0.7399 - categorical_accuracy: 0.7392 - val_loss: 1.0669 - val_categorical_accuracy: 0.6564 Epoch 35/40 34/34 [==============================] - 1s 31ms/step - loss: 0.6470 - categorical_accuracy: 0.7708 - val_loss: 1.1395 - val_categorical_accuracy: 0.6424 Epoch 36/40 34/34 [==============================] - 1s 31ms/step - loss: 0.6144 - categorical_accuracy: 0.7759 - val_loss: 1.1419 - val_categorical_accuracy: 0.6704 Epoch 37/40 34/34 [==============================] - 1s 31ms/step - loss: 0.5762 - categorical_accuracy: 0.7943 - val_loss: 1.1747 - val_categorical_accuracy: 0.6592 Epoch 38/40 34/34 [==============================] - 1s 31ms/step - loss: 0.5123 - categorical_accuracy: 0.8241 - val_loss: 1.2223 - val_categorical_accuracy: 0.5933 Epoch 39/40 34/34 [==============================] - 1s 31ms/step - loss: 0.4919 - categorical_accuracy: 0.8226 - val_loss: 1.2347 - val_categorical_accuracy: 0.6522 Epoch 40/40 34/34 [==============================] - 1s 31ms/step - loss: 0.4726 - categorical_accuracy: 0.8220 - val_loss: 1.2333 - val_categorical_accuracy: 0.6494
# plot loss
plot_loss_by_epoch(history2)
# evaluate the model
acc_model = model2.evaluate(X_val, y_val)
acc_model
23/23 [==============================] - 0s 6ms/step - loss: 1.2333 - categorical_accuracy: 0.6494
[1.2332631349563599, 0.6493688821792603]
# add scores to the dataframe
scores.loc[len(scores.index)] = ['conv32|64|pool|conv128|dense500 - padding=same', acc_model[1]]
scores
| Model | Accuracy Validation | |
|---|---|---|
| 0 | conv32|64|pool|conv128|dense500 | 0.723703 |
| 1 | conv32|64|pool|conv128|dense500 - padding=same | 0.649369 |
The model accuracy is lower than the first model.
There might possibly be more accuracy to be had with more epochs.
Next, we will try to simplify the model and see if that improves performance.
input_shape = X_train.shape[1:]
print(input_shape)
# remove one conv layer
layers = [
Conv2D(filters=64, kernel_size=3, activation='relu', input_shape=input_shape),
MaxPool2D(pool_size=(2,2), strides=(2,2)),
Conv2D(filters=128, kernel_size=3, activation='relu'),
Flatten(),
Dense(units=500, activation='relu'),
Dense(units=12, activation='softmax')
]
model3 = create_model(layers)
model3.summary()
(64, 64, 3) Model: "sequential_2" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_6 (Conv2D) (None, 62, 62, 64) 1792 _________________________________________________________________ max_pooling2d_2 (MaxPooling2 (None, 31, 31, 64) 0 _________________________________________________________________ conv2d_7 (Conv2D) (None, 29, 29, 128) 73856 _________________________________________________________________ flatten_2 (Flatten) (None, 107648) 0 _________________________________________________________________ dense_4 (Dense) (None, 500) 53824500 _________________________________________________________________ dense_5 (Dense) (None, 12) 6012 ================================================================= Total params: 53,906,160 Trainable params: 53,906,160 Non-trainable params: 0 _________________________________________________________________
history3 = model3.fit(X_train, y_train, epochs=40, batch_size=100, validation_data=(X_val, y_val))
Epoch 1/40 34/34 [==============================] - 2s 31ms/step - loss: 3.4316 - categorical_accuracy: 0.2107 - val_loss: 2.4384 - val_categorical_accuracy: 0.1529 Epoch 2/40 34/34 [==============================] - 1s 24ms/step - loss: 2.3213 - categorical_accuracy: 0.1943 - val_loss: 2.3078 - val_categorical_accuracy: 0.2244 Epoch 3/40 34/34 [==============================] - 1s 24ms/step - loss: 2.0627 - categorical_accuracy: 0.2881 - val_loss: 1.9268 - val_categorical_accuracy: 0.3380 Epoch 4/40 34/34 [==============================] - 1s 24ms/step - loss: 1.7895 - categorical_accuracy: 0.3768 - val_loss: 1.7949 - val_categorical_accuracy: 0.4011 Epoch 5/40 34/34 [==============================] - 1s 24ms/step - loss: 1.6789 - categorical_accuracy: 0.4066 - val_loss: 1.6694 - val_categorical_accuracy: 0.4208 Epoch 6/40 34/34 [==============================] - 1s 24ms/step - loss: 1.5811 - categorical_accuracy: 0.4490 - val_loss: 1.6476 - val_categorical_accuracy: 0.3913 Epoch 7/40 34/34 [==============================] - 1s 24ms/step - loss: 1.5476 - categorical_accuracy: 0.4541 - val_loss: 1.6627 - val_categorical_accuracy: 0.4137 Epoch 8/40 34/34 [==============================] - 1s 24ms/step - loss: 1.5002 - categorical_accuracy: 0.4791 - val_loss: 1.5540 - val_categorical_accuracy: 0.4698 Epoch 9/40 34/34 [==============================] - 1s 24ms/step - loss: 1.4462 - categorical_accuracy: 0.4899 - val_loss: 1.5456 - val_categorical_accuracy: 0.4867 Epoch 10/40 34/34 [==============================] - 1s 24ms/step - loss: 1.3988 - categorical_accuracy: 0.5170 - val_loss: 1.5125 - val_categorical_accuracy: 0.4923 Epoch 11/40 34/34 [==============================] - 1s 24ms/step - loss: 1.3900 - categorical_accuracy: 0.5236 - val_loss: 1.5088 - val_categorical_accuracy: 0.4797 Epoch 12/40 34/34 [==============================] - 1s 24ms/step - loss: 1.3476 - categorical_accuracy: 0.5308 - val_loss: 1.4603 - val_categorical_accuracy: 0.5133 Epoch 13/40 34/34 [==============================] - 1s 24ms/step - loss: 1.3120 - categorical_accuracy: 0.5504 - val_loss: 1.4752 - val_categorical_accuracy: 0.5302 Epoch 14/40 34/34 [==============================] - 1s 24ms/step - loss: 1.3124 - categorical_accuracy: 0.5398 - val_loss: 1.4431 - val_categorical_accuracy: 0.5203 Epoch 15/40 34/34 [==============================] - 1s 24ms/step - loss: 1.2832 - categorical_accuracy: 0.5612 - val_loss: 1.4128 - val_categorical_accuracy: 0.5217 Epoch 16/40 34/34 [==============================] - 1s 24ms/step - loss: 1.2438 - categorical_accuracy: 0.5726 - val_loss: 1.4384 - val_categorical_accuracy: 0.5119 Epoch 17/40 34/34 [==============================] - 1s 24ms/step - loss: 1.2293 - categorical_accuracy: 0.5726 - val_loss: 1.4134 - val_categorical_accuracy: 0.5302 Epoch 18/40 34/34 [==============================] - 1s 24ms/step - loss: 1.1960 - categorical_accuracy: 0.5865 - val_loss: 1.4108 - val_categorical_accuracy: 0.5330 Epoch 19/40 34/34 [==============================] - 1s 24ms/step - loss: 1.2009 - categorical_accuracy: 0.5789 - val_loss: 1.3909 - val_categorical_accuracy: 0.5231 Epoch 20/40 34/34 [==============================] - 1s 24ms/step - loss: 1.1676 - categorical_accuracy: 0.5967 - val_loss: 1.3773 - val_categorical_accuracy: 0.5288 Epoch 21/40 34/34 [==============================] - 1s 24ms/step - loss: 1.1420 - categorical_accuracy: 0.6078 - val_loss: 1.4060 - val_categorical_accuracy: 0.5302 Epoch 22/40 34/34 [==============================] - 1s 24ms/step - loss: 1.1452 - categorical_accuracy: 0.6003 - val_loss: 1.3679 - val_categorical_accuracy: 0.5442 Epoch 23/40 34/34 [==============================] - 1s 24ms/step - loss: 1.1288 - categorical_accuracy: 0.6057 - val_loss: 1.3879 - val_categorical_accuracy: 0.5568 Epoch 24/40 34/34 [==============================] - 1s 24ms/step - loss: 1.1057 - categorical_accuracy: 0.6099 - val_loss: 1.3817 - val_categorical_accuracy: 0.5428 Epoch 25/40 34/34 [==============================] - 1s 24ms/step - loss: 1.0831 - categorical_accuracy: 0.6241 - val_loss: 1.3645 - val_categorical_accuracy: 0.5358 Epoch 26/40 34/34 [==============================] - 1s 24ms/step - loss: 1.0703 - categorical_accuracy: 0.6277 - val_loss: 1.4053 - val_categorical_accuracy: 0.5316 Epoch 27/40 34/34 [==============================] - 1s 24ms/step - loss: 1.0602 - categorical_accuracy: 0.6310 - val_loss: 1.3877 - val_categorical_accuracy: 0.5554 Epoch 28/40 34/34 [==============================] - 1s 24ms/step - loss: 1.0492 - categorical_accuracy: 0.6247 - val_loss: 1.4315 - val_categorical_accuracy: 0.5344 Epoch 29/40 34/34 [==============================] - 1s 24ms/step - loss: 1.0336 - categorical_accuracy: 0.6415 - val_loss: 1.3595 - val_categorical_accuracy: 0.5512 Epoch 30/40 34/34 [==============================] - 1s 24ms/step - loss: 1.0335 - categorical_accuracy: 0.6334 - val_loss: 1.3585 - val_categorical_accuracy: 0.5624 Epoch 31/40 34/34 [==============================] - 1s 24ms/step - loss: 0.9949 - categorical_accuracy: 0.6421 - val_loss: 1.3521 - val_categorical_accuracy: 0.5820 Epoch 32/40 34/34 [==============================] - 1s 25ms/step - loss: 0.9749 - categorical_accuracy: 0.6608 - val_loss: 1.3463 - val_categorical_accuracy: 0.5568 Epoch 33/40 34/34 [==============================] - 1s 24ms/step - loss: 0.9108 - categorical_accuracy: 0.6830 - val_loss: 1.2367 - val_categorical_accuracy: 0.6199 Epoch 34/40 34/34 [==============================] - 1s 24ms/step - loss: 0.8156 - categorical_accuracy: 0.7179 - val_loss: 1.2522 - val_categorical_accuracy: 0.6045 Epoch 35/40 34/34 [==============================] - 1s 24ms/step - loss: 0.7699 - categorical_accuracy: 0.7344 - val_loss: 1.1163 - val_categorical_accuracy: 0.6564 Epoch 36/40 34/34 [==============================] - 1s 24ms/step - loss: 0.7207 - categorical_accuracy: 0.7504 - val_loss: 1.1021 - val_categorical_accuracy: 0.6690 Epoch 37/40 34/34 [==============================] - 1s 24ms/step - loss: 0.6703 - categorical_accuracy: 0.7633 - val_loss: 1.1298 - val_categorical_accuracy: 0.6410 Epoch 38/40 34/34 [==============================] - 1s 24ms/step - loss: 0.6437 - categorical_accuracy: 0.7765 - val_loss: 1.1177 - val_categorical_accuracy: 0.6508 Epoch 39/40 34/34 [==============================] - 1s 24ms/step - loss: 0.6274 - categorical_accuracy: 0.7835 - val_loss: 1.1359 - val_categorical_accuracy: 0.6171 Epoch 40/40 34/34 [==============================] - 1s 24ms/step - loss: 0.5773 - categorical_accuracy: 0.7973 - val_loss: 1.1376 - val_categorical_accuracy: 0.6592
plot_loss_by_epoch(history3)
# evaluate the model
acc_model = model3.evaluate(X_val, y_val)
acc_model
23/23 [==============================] - 0s 5ms/step - loss: 1.1376 - categorical_accuracy: 0.6592
[1.1376171112060547, 0.6591865420341492]
scores.loc[len(scores.index)] = ['conv64|pool|conv128|dense500', acc_model[1]]
scores
| Model | Accuracy Validation | |
|---|---|---|
| 0 | conv32|64|pool|conv128|dense500 | 0.723703 |
| 1 | conv32|64|pool|conv128|dense500 - padding=same | 0.649369 |
| 2 | conv64|pool|conv128|dense500 | 0.659187 |
Performance improved over previous model but is lower than the original
We will try raising the units in the dense layer as compared to the first model
# define layers and create a model
input_shape = X_train.shape[1:]
print(input_shape)
# initializer
# init = LecunNormal(seed=0)
layers = [
Conv2D(filters=32, kernel_size=3, activation='relu', input_shape=input_shape),
Conv2D(filters=64, kernel_size=3, activation='relu'),
MaxPool2D(pool_size=(2,2), strides=(2,2)),
Conv2D(filters=128, kernel_size=3, activation='relu'),
Flatten(),
Dense(units=1000),
Dense(units=12, activation='softmax')
]
model4 = create_model(layers)
model4.summary()
(64, 64, 3) Model: "sequential_3" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_8 (Conv2D) (None, 62, 62, 32) 896 _________________________________________________________________ conv2d_9 (Conv2D) (None, 60, 60, 64) 18496 _________________________________________________________________ max_pooling2d_3 (MaxPooling2 (None, 30, 30, 64) 0 _________________________________________________________________ conv2d_10 (Conv2D) (None, 28, 28, 128) 73856 _________________________________________________________________ flatten_3 (Flatten) (None, 100352) 0 _________________________________________________________________ dense_6 (Dense) (None, 1000) 100353000 _________________________________________________________________ dense_7 (Dense) (None, 12) 12012 ================================================================= Total params: 100,458,260 Trainable params: 100,458,260 Non-trainable params: 0 _________________________________________________________________
# fit the model
history4 = model4.fit(X_train, y_train, epochs=20, batch_size=100, validation_data=(X_val, y_val))
Epoch 1/20 34/34 [==============================] - 2s 41ms/step - loss: 8.1537 - categorical_accuracy: 0.2023 - val_loss: 2.4284 - val_categorical_accuracy: 0.1290 Epoch 2/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4207 - categorical_accuracy: 0.1398 - val_loss: 2.4266 - val_categorical_accuracy: 0.1164 Epoch 3/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4201 - categorical_accuracy: 0.1359 - val_loss: 2.4150 - val_categorical_accuracy: 0.1374 Epoch 4/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4180 - categorical_accuracy: 0.1377 - val_loss: 2.4139 - val_categorical_accuracy: 0.1374 Epoch 5/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4175 - categorical_accuracy: 0.1314 - val_loss: 2.4138 - val_categorical_accuracy: 0.1374 Epoch 6/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4180 - categorical_accuracy: 0.1377 - val_loss: 2.4139 - val_categorical_accuracy: 0.1374 Epoch 7/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4176 - categorical_accuracy: 0.1299 - val_loss: 2.4140 - val_categorical_accuracy: 0.1374 Epoch 8/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4170 - categorical_accuracy: 0.1377 - val_loss: 2.4137 - val_categorical_accuracy: 0.1374 Epoch 9/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4170 - categorical_accuracy: 0.1377 - val_loss: 2.4138 - val_categorical_accuracy: 0.1374 Epoch 10/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4175 - categorical_accuracy: 0.1338 - val_loss: 2.4138 - val_categorical_accuracy: 0.1374 Epoch 11/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4170 - categorical_accuracy: 0.1359 - val_loss: 2.4137 - val_categorical_accuracy: 0.1374 Epoch 12/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4173 - categorical_accuracy: 0.1377 - val_loss: 2.4139 - val_categorical_accuracy: 0.1374 Epoch 13/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4170 - categorical_accuracy: 0.1377 - val_loss: 2.4137 - val_categorical_accuracy: 0.1374 Epoch 14/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4172 - categorical_accuracy: 0.1281 - val_loss: 2.4140 - val_categorical_accuracy: 0.1374 Epoch 15/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4172 - categorical_accuracy: 0.1377 - val_loss: 2.4136 - val_categorical_accuracy: 0.1374 Epoch 16/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4178 - categorical_accuracy: 0.1317 - val_loss: 2.4140 - val_categorical_accuracy: 0.1374 Epoch 17/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4167 - categorical_accuracy: 0.1377 - val_loss: 2.4138 - val_categorical_accuracy: 0.1374 Epoch 18/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4170 - categorical_accuracy: 0.1377 - val_loss: 2.4140 - val_categorical_accuracy: 0.1374 Epoch 19/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4174 - categorical_accuracy: 0.1377 - val_loss: 2.4138 - val_categorical_accuracy: 0.1374 Epoch 20/20 34/34 [==============================] - 1s 36ms/step - loss: 2.4172 - categorical_accuracy: 0.1377 - val_loss: 2.4138 - val_categorical_accuracy: 0.1374
plot_loss_by_epoch(history4)
# evaluate the model
acc_model = model4.evaluate(X_val, y_val)
acc_model
23/23 [==============================] - 0s 7ms/step - loss: 2.4138 - categorical_accuracy: 0.1374
[2.4138476848602295, 0.13744740188121796]
scores.loc[len(scores.index)] = ['conv32|64|pool|conv128|dense1000', acc_model[1]]
scores
| Model | Accuracy Validation | |
|---|---|---|
| 0 | conv32|64|pool|conv128|dense500 | 0.723703 |
| 1 | conv32|64|pool|conv128|dense500 - padding=same | 0.649369 |
| 2 | conv64|pool|conv128|dense500 | 0.659187 |
| 3 | conv32|64|pool|conv128|dense1000 | 0.137447 |
This model got stuck.
We will train using class weights (computed previously when splitting the dataset).
# define layers and create a model
# no difference in the layers - we'll just pass class weights to the fit() function
input_shape = X_train.shape[1:]
print(input_shape)
layers = [
Conv2D(filters=32, kernel_size=3, activation='relu', input_shape=input_shape),
Conv2D(filters=64, kernel_size=3, activation='relu'),
MaxPool2D(pool_size=(2,2), strides=(2,2)),
Conv2D(filters=128, kernel_size=3, activation='relu'),
Flatten(),
Dense(units=500, activation='relu'),
Dense(units=12, activation='softmax')
]
model5 = create_model(layers)
model5.summary()
(64, 64, 3) Model: "sequential_4" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_11 (Conv2D) (None, 62, 62, 32) 896 _________________________________________________________________ conv2d_12 (Conv2D) (None, 60, 60, 64) 18496 _________________________________________________________________ max_pooling2d_4 (MaxPooling2 (None, 30, 30, 64) 0 _________________________________________________________________ conv2d_13 (Conv2D) (None, 28, 28, 128) 73856 _________________________________________________________________ flatten_4 (Flatten) (None, 100352) 0 _________________________________________________________________ dense_8 (Dense) (None, 500) 50176500 _________________________________________________________________ dense_9 (Dense) (None, 12) 6012 ================================================================= Total params: 50,275,760 Trainable params: 50,275,760 Non-trainable params: 0 _________________________________________________________________
history5 = model5.fit(X_train, y_train, epochs=40, batch_size=100, validation_data=(X_val, y_val), class_weight=weights)
Epoch 1/40 34/34 [==============================] - 2s 34ms/step - loss: 0.4292 - categorical_accuracy: 0.0815 - val_loss: 2.4854 - val_categorical_accuracy: 0.0603 Epoch 2/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2984 - categorical_accuracy: 0.0532 - val_loss: 2.4850 - val_categorical_accuracy: 0.0477 Epoch 3/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2983 - categorical_accuracy: 0.0550 - val_loss: 2.4855 - val_categorical_accuracy: 0.1052 Epoch 4/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2983 - categorical_accuracy: 0.0848 - val_loss: 2.4856 - val_categorical_accuracy: 0.1052 Epoch 5/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0950 - val_loss: 2.4855 - val_categorical_accuracy: 0.1052 Epoch 6/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0749 - val_loss: 2.4848 - val_categorical_accuracy: 0.1094 Epoch 7/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.1209 - val_loss: 2.4842 - val_categorical_accuracy: 0.1094 Epoch 8/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.1110 - val_loss: 2.4840 - val_categorical_accuracy: 0.1094 Epoch 9/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0962 - val_loss: 2.4847 - val_categorical_accuracy: 0.0813 Epoch 10/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0791 - val_loss: 2.4839 - val_categorical_accuracy: 0.1094 Epoch 11/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0686 - val_loss: 2.4845 - val_categorical_accuracy: 0.1052 Epoch 12/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0598 - val_loss: 2.4856 - val_categorical_accuracy: 0.0477 Epoch 13/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0487 - val_loss: 2.4849 - val_categorical_accuracy: 0.0477 Epoch 14/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0577 - val_loss: 2.4847 - val_categorical_accuracy: 0.0603 Epoch 15/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0605 - val_loss: 2.4846 - val_categorical_accuracy: 0.0603 Epoch 16/40 34/34 [==============================] - 1s 30ms/step - loss: 0.2982 - categorical_accuracy: 0.0605 - val_loss: 2.4852 - val_categorical_accuracy: 0.0603 Epoch 17/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.1008 - val_loss: 2.4850 - val_categorical_accuracy: 0.1052 Epoch 18/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0583 - val_loss: 2.4854 - val_categorical_accuracy: 0.0477 Epoch 19/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0574 - val_loss: 2.4854 - val_categorical_accuracy: 0.0463 Epoch 20/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0445 - val_loss: 2.4858 - val_categorical_accuracy: 0.0463 Epoch 21/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0487 - val_loss: 2.4851 - val_categorical_accuracy: 0.0463 Epoch 22/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0538 - val_loss: 2.4851 - val_categorical_accuracy: 0.0463 Epoch 23/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0466 - val_loss: 2.4853 - val_categorical_accuracy: 0.0463 Epoch 24/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0466 - val_loss: 2.4858 - val_categorical_accuracy: 0.0463 Epoch 25/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0451 - val_loss: 2.4856 - val_categorical_accuracy: 0.0463 Epoch 26/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0427 - val_loss: 2.4856 - val_categorical_accuracy: 0.0463 Epoch 27/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0716 - val_loss: 2.4850 - val_categorical_accuracy: 0.0463 Epoch 28/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0644 - val_loss: 2.4851 - val_categorical_accuracy: 0.0477 Epoch 29/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0445 - val_loss: 2.4856 - val_categorical_accuracy: 0.0463 Epoch 30/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0571 - val_loss: 2.4849 - val_categorical_accuracy: 0.0463 Epoch 31/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0716 - val_loss: 2.4848 - val_categorical_accuracy: 0.0463 Epoch 32/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0641 - val_loss: 2.4845 - val_categorical_accuracy: 0.0463 Epoch 33/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0466 - val_loss: 2.4847 - val_categorical_accuracy: 0.0463 Epoch 34/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0920 - val_loss: 2.4845 - val_categorical_accuracy: 0.1094 Epoch 35/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.1086 - val_loss: 2.4846 - val_categorical_accuracy: 0.1094 Epoch 36/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.1086 - val_loss: 2.4841 - val_categorical_accuracy: 0.1094 Epoch 37/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0773 - val_loss: 2.4847 - val_categorical_accuracy: 0.0813 Epoch 38/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0656 - val_loss: 2.4848 - val_categorical_accuracy: 0.0477 Epoch 39/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0962 - val_loss: 2.4849 - val_categorical_accuracy: 0.1094 Epoch 40/40 34/34 [==============================] - 1s 29ms/step - loss: 0.2982 - categorical_accuracy: 0.0878 - val_loss: 2.4850 - val_categorical_accuracy: 0.1094
# plot loss
plot_loss_by_epoch(history5)
# evaluate the model
acc_model = model5.evaluate(X_val, y_val)
acc_model
23/23 [==============================] - 0s 4ms/step - loss: 2.4850 - categorical_accuracy: 0.1094
[2.485004425048828, 0.10939691215753555]
scores.loc[len(scores.index)] = ['conv32|64|pool|conv128|dense1000 - w/class weights', acc_model[1]]
scores
| Model | Accuracy Validation | |
|---|---|---|
| 0 | conv32|64|pool|conv128|dense500 | 0.723703 |
| 1 | conv32|64|pool|conv128|dense500 - padding=same | 0.649369 |
| 2 | conv64|pool|conv128|dense500 | 0.659187 |
| 3 | conv32|64|pool|conv128|dense1000 | 0.137447 |
| 4 | conv32|64|pool|conv128|dense1000 - w/class wei... | 0.109397 |
This model got stuck
# use best model to evaluate against test data
model.evaluate(X_test, y_test)
23/23 [==============================] - 0s 6ms/step - loss: 1.5804 - categorical_accuracy: 0.7444
[1.5804247856140137, 0.7443820238113403]
# TODO change model to correct model name
plot_confusion_matrix(model, X_test, y_test)
precision recall f1-score support
Black-grass 0.28 0.25 0.26 40
Charlock 0.91 0.90 0.91 59
Cleavers 0.78 0.88 0.83 43
Common Chickweed 0.88 0.89 0.89 91
Common wheat 0.49 0.52 0.50 33
Fat Hen 0.72 0.65 0.68 71
Loose Silky-bent 0.61 0.69 0.65 98
Maize 0.85 0.67 0.75 33
Scentless Mayweed 0.71 0.74 0.73 77
Shepherds Purse 0.81 0.60 0.69 35
Small-flowered Cranesbill 0.89 0.92 0.91 74
Sugar beet 0.80 0.81 0.80 58
accuracy 0.74 712
macro avg 0.73 0.71 0.72 712
weighted avg 0.74 0.74 0.74 712
Performance against the test data with the initial model is close to the accuracy against the validation set with the same model.
The model generalizes well.
The confusion matrix is similar to that of validation set predictions in that similar classification errors are made.
E.g. Black-grass is often predicted to be Loose Silky-bent